#!/usr/bin/env python
# encoding: utf-8

"""
@version: 0.1
@author: Kouen
@license: Apache Licence
@email: jobkouen@outlook.com
@software: PyCharm Community Edition
@file: ow_daily.py
@time: 2016/11/3 上午11:28
@简介:

"""
import os, sys
sys.path.insert(0, os.getcwd()[:os.getcwd().find('streaming')-1])

from scrapy.contrib.spiders import Spider
import scrapy
import SimpleMonitorConfig as INI
import DevConst as Const


class ScrapyKilledItem(scrapy.Item):
     apps = scrapy.Field()


class ScrapyKilledApp(Spider):
    name = "ScrapyKilledApp"

    def start_requests(self):
        for url in INI.KilledURL:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        app_list = response.xpath('//script')[-1].xpath('text()').re(r'cluster/app/(app.*?)\'>.*</a>","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)","(.*?)"')
        app_count = int(len(app_list)/12)
        item = ScrapyKilledItem()
        item['apps'] = set()

        for index in range(app_count):
            app_detail = app_list[index*12:(index+1)*12]
            item['apps'].add((
                app_detail[Const.ID],
                app_detail[Const.NAME],
                app_detail[Const.QUEUE]
            ))

        return item
